diff --git a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml index 2953a7f52135..7bb45a26eaea 100644 --- a/.librarian/generator-input/client-post-processing/bigtable-integration.yaml +++ b/.librarian/generator-input/client-post-processing/bigtable-integration.yaml @@ -327,7 +327,6 @@ replacements: :maxdepth: 3 data_client/data_client_usage - classic_client/usage admin_client/admin_client_usage count: 1 - paths: [ @@ -340,19 +339,14 @@ replacements: .. _Product Documentation: https://cloud.google.com/bigtable/docs - Async Data Client - ------------------------- + Data Client + ----------- - :code:`v2.23.0` includes a release of the new :code:`BigtableDataClientAsync` client, accessible at the import path - :code:`google.cloud.bigtable.data`. - - The new client brings a simplified API and increased performance using asyncio. - The new client is focused on the data API (i.e. reading and writing Bigtable data), with admin operations - remaining exclusively in the existing synchronous client. - - Feedback and bug reports are welcome at cbt-python-client-v3-feedback@google.com, - or through the Github `issue tracker`_. + The modern Data Client is accessible at the import path :code:`google.cloud.bigtable.data` and provides both + synchronous (:code:`BigtableDataClient`) and asynchronous (:code:`BigtableDataClientAsync`) surfaces. + The Data Client brings a simplified API and increased performance focused on reading and writing Bigtable data. + Admin operations (such as table and instance management) are handled via :code:`google.cloud.bigtable_admin_v2`. .. note:: @@ -360,9 +354,6 @@ replacements: performance benefits, the codebase should be designed to be async from the ground up. - .. _issue tracker: https://github.com/googleapis/google-cloud-python/issues - - Quick Start count: 1 # Note: noxfile.py is heavily customized so we clobber the whole file. @@ -872,9 +863,6 @@ replacements: os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - # Customization: Add extra sections to the table of contents for the Classic vs Async clients - session.install("pyyaml") - session.run("python", "docs/scripts/patch_devsite_toc.py") @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/packages/google-cloud-bigtable/README.rst b/packages/google-cloud-bigtable/README.rst index 37d59081d319..3a950687b97a 100644 --- a/packages/google-cloud-bigtable/README.rst +++ b/packages/google-cloud-bigtable/README.rst @@ -20,19 +20,14 @@ a Google Cloud project. .. _Product Documentation: https://cloud.google.com/bigtable/docs -Async Data Client -------------------------- - -:code:`v2.23.0` includes a release of the new :code:`BigtableDataClientAsync` client, accessible at the import path -:code:`google.cloud.bigtable.data`. - -The new client brings a simplified API and increased performance using asyncio. -The new client is focused on the data API (i.e. reading and writing Bigtable data), with admin operations -remaining exclusively in the existing synchronous client. +Data Client +----------- -Feedback and bug reports are welcome at cbt-python-client-v3-feedback@google.com, -or through the Github `issue tracker`_. +The modern Data Client is accessible at the import path :code:`google.cloud.bigtable.data` and provides both +synchronous (:code:`BigtableDataClient`) and asynchronous (:code:`BigtableDataClientAsync`) surfaces. +The Data Client brings a simplified API and increased performance focused on reading and writing Bigtable data. +Admin operations (such as table and instance management) are handled via :code:`google.cloud.bigtable_admin_v2`. .. note:: @@ -40,9 +35,6 @@ or through the Github `issue tracker`_. performance benefits, the codebase should be designed to be async from the ground up. -.. _issue tracker: https://github.com/googleapis/google-cloud-python/issues - - Quick Start ----------- diff --git a/packages/google-cloud-bigtable/docs/admin_client/types_.rst b/packages/google-cloud-bigtable/docs/admin_client/types_.rst index ef32b9684bd4..d2c1af243848 100644 --- a/packages/google-cloud-bigtable/docs/admin_client/types_.rst +++ b/packages/google-cloud-bigtable/docs/admin_client/types_.rst @@ -5,6 +5,14 @@ Types for Google Cloud Bigtable Admin v2 API :members: :show-inheritance: -.. automodule:: google.cloud.bigtable_admin_v2.overlay.types +.. autoclass:: google.cloud.bigtable_admin_v2.AsyncRestoreTableOperation + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable_admin_v2.RestoreTableOperation + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable_admin_v2.WaitForConsistencyRequest :members: :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/app-profile.rst b/packages/google-cloud-bigtable/docs/classic_client/app-profile.rst deleted file mode 100644 index 5c9d426c2062..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/app-profile.rst +++ /dev/null @@ -1,6 +0,0 @@ -App Profile -~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.app_profile - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/backup.rst b/packages/google-cloud-bigtable/docs/classic_client/backup.rst deleted file mode 100644 index e75abd43143c..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/backup.rst +++ /dev/null @@ -1,6 +0,0 @@ -Backup -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.backup - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/batcher.rst b/packages/google-cloud-bigtable/docs/classic_client/batcher.rst deleted file mode 100644 index f9b9756e5414..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/batcher.rst +++ /dev/null @@ -1,13 +0,0 @@ -Mutations Batching -~~~~~~~~~~~~~~~~~~ - -.. warning:: - - If using ``MutationsBatcher``, please ensure you are using - ``google-cloud-bigtable >= 2.42.0``, or consider migrating to the - batcher provided by the synchronous/asynchronous data client - (:mod:`google.cloud.bigtable.data`). - -.. automodule:: google.cloud.bigtable.batcher - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/client-intro.rst b/packages/google-cloud-bigtable/docs/classic_client/client-intro.rst deleted file mode 100644 index 2420684996a5..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/client-intro.rst +++ /dev/null @@ -1,90 +0,0 @@ -Base for Everything -=================== - -To use the API, the :class:`Client ` -class defines a high-level interface which handles authorization -and creating other objects: - -.. code:: python - - from google.cloud.bigtable.client import Client - client = Client() - -Long-lived Defaults -------------------- - -When creating a :class:`Client `, the -``user_agent`` argument has sensible a default -(:data:`DEFAULT_USER_AGENT `). -However, you may over-ride it and the value will be used throughout all API -requests made with the ``client`` you create. - -Configuration -------------- - -- For an overview of authentication in ``google-cloud-python``, - see `Authentication `_. - -- In addition to any authentication configuration, you can also set the - :envvar:`GOOGLE_CLOUD_PROJECT` environment variable for the Google Cloud Console - project you'd like to interact with. If your code is running in Google App - Engine or Google Compute Engine the project will be detected automatically. - (Setting this environment variable is not required, you may instead pass the - ``project`` explicitly when constructing a - :class:`Client `). - -- After configuring your environment, create a - :class:`Client ` - - .. code:: - - >>> from google.cloud import bigtable - >>> client = bigtable.Client() - - or pass in ``credentials`` and ``project`` explicitly - - .. code:: - - >>> from google.cloud import bigtable - >>> client = bigtable.Client(project='my-project', credentials=creds) - -.. tip:: - - Be sure to use the **Project ID**, not the **Project Number**. - -Admin API Access ----------------- - -If you'll be using your client to make `Instance Admin`_ and `Table Admin`_ -API requests, you'll need to pass the ``admin`` argument: - -.. code:: python - - client = bigtable.Client(admin=True) - -Read-Only Mode --------------- - -If, on the other hand, you only have (or want) read access to the data, -you can pass the ``read_only`` argument: - -.. code:: python - - client = bigtable.Client(read_only=True) - -This will ensure that the -:data:`READ_ONLY_SCOPE ` is used -for API requests (so any accidental requests that would modify data will -fail). - -Next Step ---------- - -After a :class:`Client `, the next highest-level -object is an :class:`Instance `. You'll need -one before you can interact with tables or data. - -Head next to learn about the :doc:`instance-api`. - -.. _Instance Admin: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_instance_admin.proto -.. _Table Admin: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_table_admin.proto diff --git a/packages/google-cloud-bigtable/docs/classic_client/client.rst b/packages/google-cloud-bigtable/docs/classic_client/client.rst deleted file mode 100644 index c48595c8ac0b..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/client.rst +++ /dev/null @@ -1,6 +0,0 @@ -Client -~~~~~~ - -.. automodule:: google.cloud.bigtable.client - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/cluster.rst b/packages/google-cloud-bigtable/docs/classic_client/cluster.rst deleted file mode 100644 index ad33aae5e0b8..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/cluster.rst +++ /dev/null @@ -1,6 +0,0 @@ -Cluster -~~~~~~~ - -.. automodule:: google.cloud.bigtable.cluster - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/column-family.rst b/packages/google-cloud-bigtable/docs/classic_client/column-family.rst deleted file mode 100644 index de6c1eb1f5df..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/column-family.rst +++ /dev/null @@ -1,49 +0,0 @@ -Column Families -=============== - -When creating a -:class:`ColumnFamily `, it is -possible to set garbage collection rules for expired data. - -By setting a rule, cells in the table matching the rule will be deleted -during periodic garbage collection (which executes opportunistically in the -background). - -The types -:class:`MaxAgeGCRule `, -:class:`MaxVersionsGCRule `, -:class:`GarbageCollectionRuleUnion ` and -:class:`GarbageCollectionRuleIntersection ` -can all be used as the optional ``gc_rule`` argument in the -:class:`ColumnFamily ` -constructor. This value is then used in the -:meth:`create() ` and -:meth:`update() ` methods. - -These rules can be nested arbitrarily, with a -:class:`MaxAgeGCRule ` or -:class:`MaxVersionsGCRule ` -at the lowest level of the nesting: - -.. code:: python - - import datetime - - max_age = datetime.timedelta(days=3) - rule1 = MaxAgeGCRule(max_age) - rule2 = MaxVersionsGCRule(1) - - # Make a composite that matches anything older than 3 days **AND** - # with more than 1 version. - rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) - - # Make another composite that matches our previous intersection - # **OR** anything that has more than 3 versions. - rule4 = GarbageCollectionRule(max_num_versions=3) - rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) - ----- - -.. automodule:: google.cloud.bigtable.column_family - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/data-api.rst b/packages/google-cloud-bigtable/docs/classic_client/data-api.rst deleted file mode 100644 index 9b50e9ec9a8e..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/data-api.rst +++ /dev/null @@ -1,351 +0,0 @@ -Data API -======== - -.. note:: - This page describes how to use the Data API with the synchronous Bigtable client. - Examples for using the Data API with the async client can be found in the - `Getting Started Guide`_. - -.. _Getting Started Guide: https://cloud.google.com/bigtable/docs/samples-python-hello - -After creating a :class:`Table ` and some -column families, you are ready to store and retrieve data. - -Cells vs. Columns vs. Column Families -+++++++++++++++++++++++++++++++++++++ - -* As explained in the :doc:`table overview `, tables can - have many column families. -* As described below, a table can also have many rows which are - specified by row keys. -* Within a row, data is stored in a cell. A cell simply has a value (as - bytes) and a timestamp. The number of cells in each row can be - different, depending on what was stored in each row. -* Each cell lies in a column (**not** a column family). A column is really - just a more **specific** modifier within a column family. A column - can be present in every column family, in only one or anywhere in between. -* Within a column family there can be many columns. For example, within - the column family ``foo`` we could have columns ``bar`` and ``baz``. - These would typically be represented as ``foo:bar`` and ``foo:baz``. - -Modifying Data -++++++++++++++ - -Since data is stored in cells, which are stored in rows, we -use the metaphor of a **row** in classes that are used to modify -(write, update, delete) data in a -:class:`Table `. - -Direct vs. Conditional vs. Append ---------------------------------- - -There are three ways to modify data in a table, described by the -`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API -methods. - -* The **direct** way is via `MutateRow`_ which involves simply - adding, overwriting or deleting cells. The - :class:`DirectRow ` class - handles direct mutations. -* The **conditional** way is via `CheckAndMutateRow`_. This method - first checks if some filter is matched in a given row, then - applies one of two sets of mutations, depending on if a match - occurred or not. (These mutation sets are called the "true - mutations" and "false mutations".) The - :class:`ConditionalRow ` class - handles conditional mutations. -* The **append** way is via `ReadModifyWriteRow`_. This simply - appends (as bytes) or increments (as an integer) data in a presumed - existing cell in a row. The - :class:`AppendRow ` class - handles append mutations. - -Row Factory ------------ - -A single factory can be used to create any of the three row types. -To create a :class:`DirectRow `: - -.. code:: python - - row = table.row(row_key) - -Unlike the previous string values we've used before, the row key must -be ``bytes``. - -To create a :class:`ConditionalRow `, -first create a :class:`RowFilter ` and -then - -.. code:: python - - cond_row = table.row(row_key, filter_=filter_) - -To create an :class:`AppendRow ` - -.. code:: python - - append_row = table.row(row_key, append=True) - -Building Up Mutations ---------------------- - -In all three cases, a set of mutations (or two sets) are built up -on a row before they are sent off in a batch via - -.. code:: python - - row.commit() - -Direct Mutations ----------------- - -Direct mutations can be added via one of four methods - -* :meth:`set_cell() ` allows a - single value to be written to a column - - .. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp) - - If the ``timestamp`` is omitted, the current time on the Google Cloud - Bigtable server will be used when the cell is stored. - - The value can either be bytes or an integer, which will be converted to - bytes as a signed 64-bit integer. - -* :meth:`delete_cell() ` deletes - all cells (i.e. for all timestamps) in a given column - - .. code:: python - - row.delete_cell(column_family_id, column) - - Remember, this only happens in the ``row`` we are using. - - If we only want to delete cells from a limited range of time, a - :class:`TimestampRange ` can - be used - - .. code:: python - - row.delete_cell(column_family_id, column, - time_range=time_range) - -* :meth:`delete_cells() ` does - the same thing as - :meth:`delete_cell() `, - but accepts a list of columns in a column family rather than a single one. - - .. code:: python - - row.delete_cells(column_family_id, [column1, column2], - time_range=time_range) - - In addition, if we want to delete cells from every column in a column family, - the special :attr:`ALL_COLUMNS ` - value can be used - - .. code:: python - - row.delete_cells(column_family_id, row.ALL_COLUMNS, - time_range=time_range) - -* :meth:`delete() ` will delete the - entire row - - .. code:: python - - row.delete() - -Conditional Mutations ---------------------- - -Making **conditional** modifications is essentially identical -to **direct** modifications: it uses the exact same methods -to accumulate mutations. - -However, each mutation added must specify a ``state``: will the mutation be -applied if the filter matches or if it fails to match. - -For example: - -.. code:: python - - cond_row.set_cell(column_family_id, column, value, - timestamp=timestamp, state=True) - -will add to the set of true mutations. - -Append Mutations ----------------- - -Append mutations can be added via one of two methods - -* :meth:`append_cell_value() ` - appends a bytes value to an existing cell: - - .. code:: python - - append_row.append_cell_value(column_family_id, column, bytes_value) - -* :meth:`increment_cell_value() ` - increments an integer value in an existing cell: - - .. code:: python - - append_row.increment_cell_value(column_family_id, column, int_value) - - Since only bytes are stored in a cell, the cell value is decoded as - a signed 64-bit integer before being incremented. (This happens on - the Google Cloud Bigtable server, not in the library.) - -Notice that no timestamp was specified. This is because **append** mutations -operate on the latest value of the specified column. - -If there are no cells in the specified column, then the empty string (bytes -case) or zero (integer case) are the assumed values. - -Starting Fresh --------------- - -If accumulated mutations need to be dropped, use - -.. code:: python - - row.clear() - -Reading Data -++++++++++++ - -Read Single Row from a Table ----------------------------- - -To make a `ReadRows`_ API request for a single row key, use -:meth:`Table.read_row() `: - -.. code:: python - - >>> row_data = table.read_row(row_key) - >>> row_data.cells - { - u'fam1': { - b'col1': [ - , - , - ], - b'col2': [ - , - ], - }, - u'fam2': { - b'col3': [ - , - , - , - ], - }, - } - >>> cell = row_data.cells[u'fam1'][b'col1'][0] - >>> cell - - >>> cell.value - b'val1' - >>> cell.timestamp - datetime.datetime(2016, 2, 27, 3, 41, 18, 122823, tzinfo=) - -Rather than returning a :class:`DirectRow ` -or similar class, this method returns a -:class:`PartialRowData ` -instance. This class is used for reading and parsing data rather than for -modifying data (as :class:`DirectRow ` is). - -A filter can also be applied to the results: - -.. code:: python - - row_data = table.read_row(row_key, filter_=filter_val) - -The allowable ``filter_`` values are the same as those used for a -:class:`ConditionalRow `. For -more information, see the -:meth:`Table.read_row() ` documentation. - -Stream Many Rows from a Table ------------------------------ - -To make a `ReadRows`_ API request for a stream of rows, use -:meth:`Table.read_rows() `: - -.. code:: python - - row_data = table.read_rows() - -Using gRPC over HTTP/2, a continual stream of responses will be delivered. -In particular - -* :meth:`consume_next() ` - pulls the next result from the stream, parses it and stores it on the - :class:`PartialRowsData ` instance -* :meth:`consume_all() ` - pulls results from the stream until there are no more -* :meth:`cancel() ` closes - the stream - -See the :class:`PartialRowsData ` -documentation for more information. - -As with -:meth:`Table.read_row() `, an optional -``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` -can be supplied for the stream, a ``limit`` can be set and a boolean -``allow_row_interleaving`` can be specified to allow faster streamed results -at the potential cost of non-sequential reads. - -See the :meth:`Table.read_rows() ` -documentation for more information on the optional arguments. - -Sample Keys in a Table ----------------------- - -Make a `SampleRowKeys`_ API request with -:meth:`Table.sample_row_keys() `: - -.. code:: python - - keys_iterator = table.sample_row_keys() - -The returned row keys will delimit contiguous sections of the table of -approximately equal size, which can be used to break up the data for -distributed tasks like mapreduces. - -As with -:meth:`Table.read_rows() `, the -returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. - -The next key in the result can be accessed via - -.. code:: python - - next_key = keys_iterator.next() - -or all keys can be iterated over via - -.. code:: python - - for curr_key in keys_iterator: - do_something(curr_key) - -Just as with reading, the stream can be canceled: - -.. code:: python - - keys_iterator.cancel() - -.. _ReadRows: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L42-L72 -.. _SampleRowKeys: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L184-L199 -.. _MutateRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L230-L256 -.. _CheckAndMutateRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L339-L386 -.. _ReadModifyWriteRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L401-L430 diff --git a/packages/google-cloud-bigtable/docs/classic_client/encryption-info.rst b/packages/google-cloud-bigtable/docs/classic_client/encryption-info.rst deleted file mode 100644 index 46f19880fcac..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/encryption-info.rst +++ /dev/null @@ -1,6 +0,0 @@ -Encryption Info -~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.encryption_info - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/instance-api.rst b/packages/google-cloud-bigtable/docs/classic_client/instance-api.rst deleted file mode 100644 index 88b4eb4dc914..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/instance-api.rst +++ /dev/null @@ -1,130 +0,0 @@ -Instance Admin API -================== - -After creating a :class:`Client `, you can -interact with individual instances for a project. - -List Instances --------------- - -If you want a comprehensive list of all existing instances, make a -`ListInstances`_ API request with -:meth:`Client.list_instances() `: - -.. code:: python - - instances = client.list_instances() - -Instance Factory ----------------- - -To create an :class:`Instance ` object: - -.. code:: python - - instance = client.instance(instance_id, display_name=display_name) - -- ``display_name`` is optional. When not provided, ``display_name`` defaults - to the ``instance_id`` value. - -You can also use :meth:`Client.instance` to create a local wrapper for -instances that have already been created with the API, or through the web -console: - -.. code:: python - - instance = client.instance(existing_instance_id) - instance.reload() - -Create a new Instance ---------------------- - -After creating the instance object, make a `CreateInstance`_ API request -with :meth:`create() `: - -.. code:: python - - instance.display_name = 'My very own instance' - instance.create() - -Check on Current Operation --------------------------- - -.. note:: - - When modifying an instance (via a `CreateInstance`_ request), the Bigtable - API will return a `long-running operation`_ and a corresponding - :class:`Operation ` object - will be returned by - :meth:`create() `. - -You can check if a long-running operation (for a -:meth:`create() ` has finished -by making a `GetOperation`_ request with -:meth:`Operation.finished() `: - -.. code:: python - - >>> operation = instance.create() - >>> operation.finished() - True - -.. note:: - - Once an :class:`Operation ` object - has returned :data:`True` from - :meth:`finished() `, the - object should not be re-used. Subsequent calls to - :meth:`finished() ` - will result in a :class:`ValueError `. - -Get metadata for an existing Instance -------------------------------------- - -After creating the instance object, make a `GetInstance`_ API request -with :meth:`reload() `: - -.. code:: python - - instance.reload() - -This will load ``display_name`` for the existing ``instance`` object. - -Update an existing Instance ---------------------------- - -After creating the instance object, make an `UpdateInstance`_ API request -with :meth:`update() `: - -.. code:: python - - instance.display_name = 'New display_name' - instance.update() - -Delete an existing Instance ---------------------------- - -Make a `DeleteInstance`_ API request with -:meth:`delete() `: - -.. code:: python - - instance.delete() - -Next Step ---------- - -Now we go down the hierarchy from -:class:`Instance ` to a -:class:`Table `. - -Head next to learn about the :doc:`table-api`. - -.. _Instance Admin API: https://cloud.google.com/bigtable/docs/creating-instance -.. _CreateInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#create-a-new-instance -.. _GetInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#get-metadata-for-an-existing-instance -.. _UpdateInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#update-an-existing-instance -.. _DeleteInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#delete-an-existing-instance -.. _ListInstances: https://googleapis.dev/python/bigtable/latest/instance-api.html#list-instances -.. _GetOperation: https://googleapis.dev/python/bigtable/latest/instance-api.html#check-on-current-operation -.. _long-running operation: https://github.com/googleapis/googleapis/blob/main/google/longrunning/operations.proto#L128-L162 diff --git a/packages/google-cloud-bigtable/docs/classic_client/instance.rst b/packages/google-cloud-bigtable/docs/classic_client/instance.rst deleted file mode 100644 index f9be9672fc64..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/instance.rst +++ /dev/null @@ -1,6 +0,0 @@ -Instance -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.instance - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/row-data.rst b/packages/google-cloud-bigtable/docs/classic_client/row-data.rst deleted file mode 100644 index 503f9b1cbdfd..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/row-data.rst +++ /dev/null @@ -1,6 +0,0 @@ -Row Data -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.row_data - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/row-filters.rst b/packages/google-cloud-bigtable/docs/classic_client/row-filters.rst deleted file mode 100644 index 9884ce400d52..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/row-filters.rst +++ /dev/null @@ -1,67 +0,0 @@ -Bigtable Row Filters -==================== - -It is possible to use a -:class:`RowFilter ` -when adding mutations to a -:class:`ConditionalRow ` and when -reading row data with :meth:`read_row() ` -or :meth:`read_rows() `. - -As laid out in the `RowFilter definition`_, the following basic filters -are provided: - -* :class:`SinkFilter <.row_filters.SinkFilter>` -* :class:`PassAllFilter <.row_filters.PassAllFilter>` -* :class:`BlockAllFilter <.row_filters.BlockAllFilter>` -* :class:`RowKeyRegexFilter <.row_filters.RowKeyRegexFilter>` -* :class:`RowSampleFilter <.row_filters.RowSampleFilter>` -* :class:`FamilyNameRegexFilter <.row_filters.FamilyNameRegexFilter>` -* :class:`ColumnQualifierRegexFilter <.row_filters.ColumnQualifierRegexFilter>` -* :class:`TimestampRangeFilter <.row_filters.TimestampRangeFilter>` -* :class:`ColumnRangeFilter <.row_filters.ColumnRangeFilter>` -* :class:`ValueRegexFilter <.row_filters.ValueRegexFilter>` -* :class:`ValueRangeFilter <.row_filters.ValueRangeFilter>` -* :class:`CellsRowOffsetFilter <.row_filters.CellsRowOffsetFilter>` -* :class:`CellsRowLimitFilter <.row_filters.CellsRowLimitFilter>` -* :class:`CellsColumnLimitFilter <.row_filters.CellsColumnLimitFilter>` -* :class:`StripValueTransformerFilter <.row_filters.StripValueTransformerFilter>` -* :class:`ApplyLabelFilter <.row_filters.ApplyLabelFilter>` - -In addition, these filters can be combined into composite filters with - -* :class:`RowFilterChain <.row_filters.RowFilterChain>` -* :class:`RowFilterUnion <.row_filters.RowFilterUnion>` -* :class:`ConditionalRowFilter <.row_filters.ConditionalRowFilter>` - -These rules can be nested arbitrarily, with a basic filter at the lowest -level. For example: - -.. code:: python - - # Filter in a specified column (matching any column family). - col1_filter = ColumnQualifierRegexFilter(b'columnbia') - - # Create a filter to label results. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - - # Combine the filters to label all the cells in columnbia. - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Create a similar filter to label cells blue. - col2_filter = ColumnQualifierRegexFilter(b'columnseeya') - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - ----- - -.. automodule:: google.cloud.bigtable.row_filters - :members: - :show-inheritance: - -.. _RowFilter definition: https://googleapis.dev/python/bigtable/latest/row-filters.html?highlight=rowfilter#google.cloud.bigtable.row_filters.RowFilter diff --git a/packages/google-cloud-bigtable/docs/classic_client/row-set.rst b/packages/google-cloud-bigtable/docs/classic_client/row-set.rst deleted file mode 100644 index 5f7a16a029ed..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/row-set.rst +++ /dev/null @@ -1,6 +0,0 @@ -Row Set -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.row_set - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/row.rst b/packages/google-cloud-bigtable/docs/classic_client/row.rst deleted file mode 100644 index 33686608b363..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/row.rst +++ /dev/null @@ -1,7 +0,0 @@ -Bigtable Row -============ - -.. automodule:: google.cloud.bigtable.row - :members: - :show-inheritance: - :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/classic_client/snippets.py b/packages/google-cloud-bigtable/docs/classic_client/snippets.py deleted file mode 100644 index 3f35e0cf536c..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/snippets.py +++ /dev/null @@ -1,781 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testable usage examples for Google Cloud Bigtable API wrapper - -Each example function takes a ``client`` argument (which must be an instance -of :class:`google.cloud.bigtable.client.Client`) and uses it to perform a task -with the API. - -To facilitate running the examples as system tests, each example is also passed -a ``to_delete`` list; the function adds to the list any objects created which -need to be deleted during teardown. - -.. note:: - This file is under progress and will be updated with more guidance from - the team. Unit tests will be added with guidance from the team. - -""" - -from datetime import datetime, timezone - -import pytest -from google.api_core.exceptions import ( - DeadlineExceeded, - NotFound, - ServiceUnavailable, - TooManyRequests, -) -from test_utils.retry import RetryErrors -from test_utils.system import unique_resource_id - -from google.cloud.bigtable import Client, enums - -UNIQUE_SUFFIX = unique_resource_id("-") -INSTANCE_ID = "snippet-tests" + UNIQUE_SUFFIX -CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX -APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX -TABLE_ID = "tabl-1" + UNIQUE_SUFFIX -ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY -LOCATION_ID = "us-central1-f" -ALT_LOCATION_ID = "us-central1-a" -PRODUCTION = enums.Instance.Type.PRODUCTION -SERVER_NODES = 3 -STORAGE_TYPE = enums.StorageType.SSD -LABEL_KEY = "python-snippet" -LABEL_STAMP = ( - datetime.now(timezone.utc).replace(microsecond=0).strftime("%Y-%m-%dt%H-%M-%S") -) -LABELS = {LABEL_KEY: str(LABEL_STAMP)} -INSTANCES_TO_DELETE = [] - -retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) -retry_504 = RetryErrors(DeadlineExceeded, max_tries=4) - - -class Config(object): - """Run-time configuration to be modified at set-up. - - This is a mutable stand-in to allow test set-up to modify - global state. - """ - - CLIENT = None - INSTANCE = None - TABLE = None - - -def setup_module(): - client = Config.CLIENT = Client(admin=True) - Config.INSTANCE = client.instance( - INSTANCE_ID, instance_type=PRODUCTION, labels=LABELS - ) - cluster = Config.INSTANCE.cluster( - CLUSTER_ID, - location_id=LOCATION_ID, - serve_nodes=SERVER_NODES, - default_storage_type=STORAGE_TYPE, - ) - operation = Config.INSTANCE.create(clusters=[cluster]) - # We want to make sure the operation completes. - operation.result(timeout=100) - Config.TABLE = Config.INSTANCE.table(TABLE_ID) - retry_504(Config.TABLE.create)() - - -def teardown_module(): - retry_429_503(Config.INSTANCE.delete)() - - for instance in INSTANCES_TO_DELETE: - try: - retry_429_503(instance.delete)() - except NotFound: - pass - - -def test_bigtable_create_instance(): - # [START bigtable_api_create_prod_instance] - from google.cloud.bigtable import Client, enums - - my_instance_id = "inst-my-" + UNIQUE_SUFFIX - my_cluster_id = "clus-my-" + UNIQUE_SUFFIX - location_id = "us-central1-f" - serve_nodes = 1 - storage_type = enums.StorageType.SSD - production = enums.Instance.Type.PRODUCTION - labels = {"prod-label": "prod-label"} - - client = Client(admin=True) - instance = client.instance(my_instance_id, instance_type=production, labels=labels) - cluster = instance.cluster( - my_cluster_id, - location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type, - ) - operation = instance.create(clusters=[cluster]) - - # We want to make sure the operation completes. - operation.result(timeout=100) - - # [END bigtable_api_create_prod_instance] - - try: - assert instance.exists() - finally: - retry_429_503(instance.delete)() - - -def test_bigtable_create_additional_cluster(): - # [START bigtable_api_create_cluster] - from google.cloud.bigtable import Client, enums - - # Assuming that there is an existing instance with `INSTANCE_ID` - # on the server already. - # to create an instance see - # 'https://cloud.google.com/bigtable/docs/creating-instance' - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - cluster_id = "clus-my-" + UNIQUE_SUFFIX - location_id = "us-central1-a" - serve_nodes = 1 - storage_type = enums.StorageType.SSD - - cluster = instance.cluster( - cluster_id, - location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type, - ) - operation = cluster.create() - # We want to make sure the operation completes. - operation.result(timeout=100) - # [END bigtable_api_create_cluster] - - try: - assert cluster.exists() - finally: - retry_429_503(cluster.delete)() - - -def test_bigtable_create_reload_delete_app_profile(): - import re - - # [START bigtable_api_create_app_profile] - from google.cloud.bigtable import Client, enums - - routing_policy_type = enums.RoutingPolicyType.ANY - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - description = "routing policy-multy" - - app_profile = instance.app_profile( - app_profile_id=APP_PROFILE_ID, - routing_policy_type=routing_policy_type, - description=description, - cluster_id=CLUSTER_ID, - ) - - app_profile = app_profile.create(ignore_warnings=True) - # [END bigtable_api_create_app_profile] - - # [START bigtable_api_app_profile_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile_name = app_profile.name - # [END bigtable_api_app_profile_name] - _profile_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/" - r"appProfiles/(?P" - r"[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _profile_name_re.match(app_profile_name) - - # [START bigtable_api_app_profile_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile_exists = app_profile.exists() - # [END bigtable_api_app_profile_exists] - assert app_profile_exists - - # [START bigtable_api_reload_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile.reload() - # [END bigtable_api_reload_app_profile] - assert app_profile.routing_policy_type == ROUTING_POLICY_TYPE - - # [START bigtable_api_update_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - app_profile.reload() - - description = "My new app profile" - app_profile.description = description - app_profile.update() - # [END bigtable_api_update_app_profile] - assert app_profile.description == description - - # [START bigtable_api_delete_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - app_profile.reload() - - app_profile.delete(ignore_warnings=True) - # [END bigtable_api_delete_app_profile] - assert not app_profile.exists() - - -def test_bigtable_list_instances(): - # [START bigtable_api_list_instances] - from google.cloud.bigtable import Client - - client = Client(admin=True) - (instances_list, failed_locations_list) = client.list_instances() - # [END bigtable_api_list_instances] - - assert len(instances_list) > 0 - - -def test_bigtable_list_clusters_on_instance(): - # [START bigtable_api_list_clusters_on_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - (clusters_list, failed_locations_list) = instance.list_clusters() - # [END bigtable_api_list_clusters_on_instance] - - assert len(clusters_list) > 0 - - -def test_bigtable_list_clusters_in_project(): - # [START bigtable_api_list_clusters_in_project] - from google.cloud.bigtable import Client - - client = Client(admin=True) - (clusters_list, failed_locations_list) = client.list_clusters() - # [END bigtable_api_list_clusters_in_project] - - assert len(clusters_list) > 0 - - -def test_bigtable_list_app_profiles(): - app_profile = Config.INSTANCE.app_profile( - app_profile_id="app-prof-" + UNIQUE_SUFFIX, - routing_policy_type=enums.RoutingPolicyType.ANY, - ) - app_profile = app_profile.create(ignore_warnings=True) - - # [START bigtable_api_list_app_profiles] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - app_profiles_list = instance.list_app_profiles() - # [END bigtable_api_list_app_profiles] - - try: - assert len(app_profiles_list) > 0 - finally: - retry_429_503(app_profile.delete)(ignore_warnings=True) - - -def test_bigtable_instance_exists(): - # [START bigtable_api_check_instance_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_exists = instance.exists() - # [END bigtable_api_check_instance_exists] - - assert instance_exists - - -def test_bigtable_cluster_exists(): - # [START bigtable_api_check_cluster_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_exists = cluster.exists() - # [END bigtable_api_check_cluster_exists] - - assert cluster_exists - - -def test_bigtable_reload_instance(): - # [START bigtable_api_reload_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - # [END bigtable_api_reload_instance] - - assert instance.type_ == PRODUCTION.value - - -def test_bigtable_reload_cluster(): - # [START bigtable_api_reload_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster.reload() - # [END bigtable_api_reload_cluster] - - assert cluster.serve_nodes == SERVER_NODES - - -def test_bigtable_update_instance(): - # [START bigtable_api_update_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - display_name = "My new instance" - instance.display_name = display_name - instance.update() - # [END bigtable_api_update_instance] - - assert instance.display_name == display_name - - -def test_bigtable_update_cluster(): - # [START bigtable_api_update_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster.serve_nodes = 4 - cluster.update() - # [END bigtable_api_update_cluster] - - assert cluster.serve_nodes == 4 - - -def test_bigtable_cluster_disable_autoscaling(): - # [START bigtable_api_cluster_disable_autoscaling] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - # Create a cluster with autoscaling enabled - cluster = instance.cluster( - CLUSTER_ID, min_serve_nodes=1, max_serve_nodes=2, cpu_utilization_percent=10 - ) - instance.create(clusters=[cluster]) - - # Disable autoscaling - cluster.disable_autoscaling(serve_nodes=4) - # [END bigtable_api_cluster_disable_autoscaling] - - assert cluster.serve_nodes == 4 - - -def test_bigtable_create_table(): - # [START bigtable_api_create_table] - from google.api_core import exceptions, retry - - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_my") - # Define the GC policy to retain only the most recent 2 versions. - max_versions_rule = column_family.MaxVersionsGCRule(2) - - # Could include other retriable exception types - # Could configure deadline, etc. - predicate_504 = retry.if_exception_type(exceptions.DeadlineExceeded) - retry_504 = retry.Retry(predicate_504) - - retry_504(table.create)(column_families={"cf1": max_versions_rule}) - # [END bigtable_api_create_table] - - try: - assert table.exists() - finally: - retry_429_503(table.delete)() - - -def test_bigtable_list_tables(): - # [START bigtable_api_list_tables] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - tables_list = instance.list_tables() - # [END bigtable_api_list_tables] - - # Check if returned list has expected table - table_names = [table.name for table in tables_list] - assert Config.TABLE.name in table_names - - -def test_bigtable_delete_cluster(): - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster_id = "clus-my-" + UNIQUE_SUFFIX - serve_nodes = 1 - cluster = instance.cluster( - cluster_id, - location_id=ALT_LOCATION_ID, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - operation = cluster.create() - # We want to make sure the operation completes. - operation.result(timeout=1000) - - # [START bigtable_api_delete_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster_to_delete = instance.cluster(cluster_id) - - cluster_to_delete.delete() - # [END bigtable_api_delete_cluster] - - assert not cluster_to_delete.exists() - - -def test_bigtable_delete_instance(): - from google.cloud.bigtable import Client - - client = Client(admin=True) - - instance_id = "snipt-inst-del" + UNIQUE_SUFFIX - instance = client.instance(instance_id, instance_type=PRODUCTION, labels=LABELS) - serve_nodes = 1 - cluster = instance.cluster( - "clus-to-delete" + UNIQUE_SUFFIX, - location_id=ALT_LOCATION_ID, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - operation = instance.create(clusters=[cluster]) - - # We want to make sure the operation completes. - operation.result(timeout=100) - - # Make sure this instance gets deleted after the test case. - INSTANCES_TO_DELETE.append(instance) - - # [START bigtable_api_delete_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - - instance_to_delete = client.instance(instance_id) - instance_to_delete.delete() - # [END bigtable_api_delete_instance] - - assert not instance_to_delete.exists() - - # Skip deleting it during module teardown if the assertion succeeds. - INSTANCES_TO_DELETE.remove(instance) - - -def test_bigtable_test_iam_permissions(): - # [START bigtable_api_test_iam_permissions] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - permissions = ["bigtable.clusters.create", "bigtable.tables.create"] - permissions_allowed = instance.test_iam_permissions(permissions) - # [END bigtable_api_test_iam_permissions] - - assert permissions_allowed == permissions - - -def test_bigtable_set_iam_policy_then_get_iam_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_set_iam_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - # [END bigtable_api_set_iam_policy] - - assert len(policy_latest.bigtable_admins) > 0 - - # [START bigtable_api_get_iam_policy] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - policy = instance.get_iam_policy() - # [END bigtable_api_get_iam_policy] - - assert len(policy.bigtable_admins) > 0 - - -def test_bigtable_project_path(): - import re - - # [START bigtable_api_project_path] - from google.cloud.bigtable import Client - - client = Client(admin=True) - project_path = client.project_path - # [END bigtable_api_project_path] - - -def test_bigtable_table_data_client(): - # [START bigtable_api_table_data_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - table_data_client = client.table_data_client - # [END bigtable_api_table_data_client] - - -def test_bigtable_table_admin_client(): - # [START bigtable_api_table_admin_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - table_admin_client = client.table_admin_client - # [END bigtable_api_table_admin_client] - - -def test_bigtable_instance_admin_client(): - # [START bigtable_api_instance_admin_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance_admin_client = client.instance_admin_client - # [END bigtable_api_instance_admin_client] - - -def test_bigtable_admins_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_admins_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_admins - # [END bigtable_api_admins_policy] - - assert len(policy) > 0 - - -def test_bigtable_readers_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_readers_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_READER_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_READER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_readers - # [END bigtable_api_readers_policy] - - assert len(policy) > 0 - - -def test_bigtable_users_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_users_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_USER_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_USER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_users - # [END bigtable_api_users_policy] - - assert len(policy) > 0 - - -def test_bigtable_viewers_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_viewers_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_VIEWER_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_VIEWER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_viewers - # [END bigtable_api_viewers_policy] - - assert len(policy) > 0 - - -def test_bigtable_instance_name(): - import re - - # [START bigtable_api_instance_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_name = instance.name - # [END bigtable_api_instance_name] - - -def test_bigtable_cluster_name(): - import re - - # [START bigtable_api_cluster_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_name = cluster.name - # [END bigtable_api_cluster_name] - - -def test_bigtable_instance_from_pb(): - # [START bigtable_api_instance_from_pb] - from google.cloud.bigtable import Client - from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2 - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - name = instance.name - instance_pb = data_v2_pb2.Instance( - name=name, display_name=INSTANCE_ID, type=PRODUCTION, labels=LABELS - ) - - instance2 = instance.from_pb(instance_pb, client) - # [END bigtable_api_instance_from_pb] - - assert instance2.name == instance.name - - -def test_bigtable_cluster_from_pb(): - # [START bigtable_api_cluster_from_pb] - from google.cloud.bigtable import Client - from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2 - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - - name = cluster.name - cluster_state = cluster.state - serve_nodes = 1 - cluster_pb = data_v2_pb2.Cluster( - name=name, - location=LOCATION_ID, - state=cluster_state, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - - cluster2 = cluster.from_pb(cluster_pb, instance) - # [END bigtable_api_cluster_from_pb] - - assert cluster2.name == cluster.name - - -def test_bigtable_instance_state(): - # [START bigtable_api_instance_state] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_state = instance.state - # [END bigtable_api_instance_state] - - assert not instance_state - - -def test_bigtable_cluster_state(): - # [START bigtable_api_cluster_state] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_state = cluster.state - # [END bigtable_api_cluster_state] - - assert not cluster_state - - -if __name__ == "__main__": - pytest.main() diff --git a/packages/google-cloud-bigtable/docs/classic_client/snippets_table.py b/packages/google-cloud-bigtable/docs/classic_client/snippets_table.py deleted file mode 100644 index 34b0255e300f..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/snippets_table.py +++ /dev/null @@ -1,1322 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testable usage examples for Google Cloud Bigtable API wrapper - -Each example function takes a ``client`` argument (which must be an instance -of :class:`google.cloud.bigtable.client.Client`) and uses it to perform a task -with the API. - -To facilitate running the examples as system tests, each example is also passed -a ``to_delete`` list; the function adds to the list any objects created which -need to be deleted during teardown. - -.. note:: - This file is under progress and will be updated with more guidance from - the team. Unit tests will be added with guidance from the team. - -""" - -from datetime import datetime, timezone - -import pytest -from google.api_core.exceptions import ServiceUnavailable, TooManyRequests -from test_utils.retry import RetryErrors -from test_utils.system import unique_resource_id - -from google.cloud.bigtable import Client, column_family, enums - -INSTANCE_ID = "snippet" + unique_resource_id("-") -CLUSTER_ID = "clus-1" + unique_resource_id("-") -TABLE_ID = "tabl-1" + unique_resource_id("-") -COLUMN_FAMILY_ID = "col_fam_id-" + unique_resource_id("-") -LOCATION_ID = "us-central1-f" -ALT_LOCATION_ID = "us-central1-a" -PRODUCTION = enums.Instance.Type.PRODUCTION -SERVER_NODES = 3 -STORAGE_TYPE = enums.StorageType.SSD -LABEL_KEY = "python-snippet" -LABEL_STAMP = ( - datetime.now(timezone.utc).replace(microsecond=0).strftime("%Y-%m-%dt%H-%M-%S") -) -LABELS = {LABEL_KEY: str(LABEL_STAMP)} -COLUMN_FAMILY_ID = "col_fam_id1" -COL_NAME1 = b"col-name1" -CELL_VAL1 = b"cell-val" -ROW_KEY1 = b"row_key_id1" -COLUMN_FAMILY_ID2 = "col_fam_id2" -COL_NAME2 = b"col-name2" -CELL_VAL2 = b"cell-val2" -ROW_KEY2 = b"row_key_id2" - -retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) - - -class Config(object): - """Run-time configuration to be modified at set-up. - - This is a mutable stand-in to allow test set-up to modify - global state. - """ - - CLIENT = None - INSTANCE = None - TABLE = None - - -def setup_module(): - client = Config.CLIENT = Client(admin=True) - Config.INSTANCE = client.instance( - INSTANCE_ID, instance_type=PRODUCTION, labels=LABELS - ) - cluster = Config.INSTANCE.cluster( - CLUSTER_ID, - location_id=LOCATION_ID, - serve_nodes=SERVER_NODES, - default_storage_type=STORAGE_TYPE, - ) - operation = Config.INSTANCE.create(clusters=[cluster]) - # We want to make sure the operation completes. - operation.result(timeout=100) - Config.TABLE = Config.INSTANCE.table(TABLE_ID) - Config.TABLE.create() - gc_rule = column_family.MaxVersionsGCRule(2) - column_family1 = Config.TABLE.column_family(COLUMN_FAMILY_ID, gc_rule=gc_rule) - column_family1.create() - gc_rule2 = column_family.MaxVersionsGCRule(4) - column_family2 = Config.TABLE.column_family(COLUMN_FAMILY_ID2, gc_rule=gc_rule2) - column_family2.create() - - -def teardown_module(): - retry_429_503(Config.INSTANCE.delete)() - - -def test_bigtable_create_table(): - # [START bigtable_api_create_table] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - # Create table without Column families. - table1 = instance.table("table_id1") - table1.create() - - # Create table with Column families. - table2 = instance.table("table_id2") - # Define the GC policy to retain only the most recent 2 versions. - max_versions_rule = column_family.MaxVersionsGCRule(2) - table2.create(column_families={"cf1": max_versions_rule}) - - # [END bigtable_api_create_table] - assert table1.exists() - assert table2.exists() - table1.delete() - table2.delete() - - -def test_bigtable_sample_row_keys(): - table_sample = Config.INSTANCE.table("table_id1_samplerow") - initial_split_keys = [b"split_key_1", b"split_key_10", b"split_key_20"] - table_sample.create(initial_split_keys=initial_split_keys) - assert table_sample.exists() - - # [START bigtable_api_sample_row_keys] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table("table_id1_samplerow") - data = table.sample_row_keys() - actual_keys, offset = zip(*[(rk.row_key, rk.offset_bytes) for rk in data]) - # [END bigtable_api_sample_row_keys] - initial_split_keys.append(b"") - assert list(actual_keys) == initial_split_keys - table.delete() - - -def test_bigtable_write_read_drop_truncate(): - # [START bigtable_api_mutate_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_20", - b"row_key_22", - b"row_key_200", - ] - col_name = b"col-name1" - rows = [] - for i, row_key in enumerate(row_keys): - value = "value_{}".format(i).encode() - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, col_name, value, timestamp=datetime.now(timezone.utc) - ) - rows.append(row) - response = table.mutate_rows(rows) - # validate that all rows written successfully - for i, status in enumerate(response): - if status.code != 0: - print("Row number {} failed to write".format(i)) - # [END bigtable_api_mutate_rows] - assert len(response) == len(rows) - # [START bigtable_api_read_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row = table.read_row(row_key) - # [END bigtable_api_read_row] - assert row.row_key.decode("utf-8") == row_key - # [START bigtable_api_read_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Read full table - partial_rows = table.read_rows() - - # Read row's value - total_rows = [] - for row in partial_rows: - cell = row.cells[COLUMN_FAMILY_ID][col_name][0] - print(cell.value.decode("utf-8")) - total_rows.append(cell) - # [END bigtable_api_read_rows] - assert len(total_rows) == len(rows) - # [START bigtable_api_drop_by_prefix] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key_prefix = b"row_key_2" - table.drop_by_prefix(row_key_prefix, timeout=200) - # [END bigtable_api_drop_by_prefix] - dropped_row_keys = [b"row_key_2", b"row_key_20", b"row_key_22", b"row_key_200"] - for row in table.read_rows(): - assert row.row_key.decode("utf-8") not in dropped_row_keys - - # [START bigtable_api_truncate_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - table.truncate(timeout=200) - # [END bigtable_api_truncate_table] - rows_data_after_truncate = [] - for row in table.read_rows(): - rows_data_after_truncate.append(row.row_key) - assert rows_data_after_truncate == [] - - -def test_bigtable_mutations_batcher(): - # [START bigtable_api_mutations_batcher] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - batcher = table.mutations_batcher() - # [END bigtable_api_mutations_batcher] - - # Below code will be used while creating batcher.py snippets. - # So not removing this code as of now. - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_20", - b"row_key_22", - b"row_key_200", - ] - column_name = "column_name".encode() - # Add a single row - row_key = row_keys[0] - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, column_name, "value-0", timestamp=datetime.now(timezone.utc) - ) - batcher.mutate(row) - # Add a collections of rows - rows = [] - for i in range(1, len(row_keys)): - row = table.row(row_keys[i]) - value = "value_{}".format(i).encode() - row.set_cell( - COLUMN_FAMILY_ID, column_name, value, timestamp=datetime.now(timezone.utc) - ) - rows.append(row) - batcher.mutate_rows(rows) - # batcher will flush current batch if it - # reaches the max flush_count - - # Manually send the current batch to Cloud Bigtable - batcher.flush() - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == len(row_keys) - table.truncate(timeout=200) - - -def test_bigtable_table_column_family(): - # [START bigtable_api_table_column_family] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table(TABLE_ID) - column_family_obj = table.column_family(COLUMN_FAMILY_ID) - # [END bigtable_api_table_column_family] - - assert column_family_obj.column_family_id == COLUMN_FAMILY_ID - - -def test_bigtable_list_tables(): - # [START bigtable_api_list_tables] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - tables_list = instance.list_tables() - # [END bigtable_api_list_tables] - assert len(tables_list) != 0 - - -def test_bigtable_table_name(): - import re - - # [START bigtable_api_table_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table(TABLE_ID) - table_name = table.name - # [END bigtable_api_table_name] - _table_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/tables/" - r"(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _table_name_re.match(table_name) - - -def test_bigtable_list_column_families(): - # [START bigtable_api_list_column_families] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_list = table.list_column_families() - # [END bigtable_api_list_column_families] - - assert len(column_family_list) > 0 - - -def test_bigtable_get_cluster_states(): - # [START bigtable_api_get_cluster_states] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - get_cluster_states = table.get_cluster_states() - # [END bigtable_api_get_cluster_states] - - assert CLUSTER_ID in get_cluster_states - - -def test_bigtable_table_test_iam_permissions(): - table_policy = Config.INSTANCE.table("table_id_iam_policy") - table_policy.create() - assert table_policy.exists - - # [START bigtable_api_table_test_iam_permissions] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - - permissions = ["bigtable.tables.mutateRows", "bigtable.tables.readRows"] - permissions_allowed = table.test_iam_permissions(permissions) - # [END bigtable_api_table_test_iam_permissions] - assert permissions_allowed == permissions - - -def test_bigtable_table_set_iam_policy_then_get_iam_policy(): - table_policy = Config.INSTANCE.table("table_id_iam_policy") - assert table_policy.exists - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_table_set_iam_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE, Policy - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = table.set_iam_policy(new_policy) - # [END bigtable_api_table_set_iam_policy] - assert len(policy_latest.bigtable_admins) > 0 - - # [START bigtable_api_table_get_iam_policy] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - policy = table.get_iam_policy() - # [END bigtable_api_table_get_iam_policy] - assert len(policy.bigtable_admins) > 0 - - -def test_bigtable_table_exists(): - # [START bigtable_api_check_table_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - table_exists = table.exists() - # [END bigtable_api_check_table_exists] - assert table_exists - - -def test_bigtable_delete_table(): - table_del = Config.INSTANCE.table("table_id_del") - table_del.create() - assert table_del.exists() - - # [START bigtable_api_delete_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_del") - - table.delete() - # [END bigtable_api_delete_table] - assert not table.exists() - - -def test_bigtable_table_row(): - # [START bigtable_api_table_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.row(row_keys[0]) - row2_obj = table.row(row_keys[1]) - # [END bigtable_api_table_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_append_row(): - # [START bigtable_api_table_append_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.append_row(row_keys[0]) - row2_obj = table.append_row(row_keys[1]) - # [END bigtable_api_table_append_row] - - row1_obj.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_direct_row(): - # [START bigtable_api_table_direct_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.direct_row(row_keys[0]) - row2_obj = table.direct_row(row_keys[1]) - # [END bigtable_api_table_direct_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_conditional_row(): - # [START bigtable_api_table_conditional_row] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_filters import PassAllFilter - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - filter_ = PassAllFilter(True) - row1_obj = table.conditional_row(row_keys[0], filter_=filter_) - row2_obj = table.conditional_row(row_keys[1], filter_=filter_) - # [END bigtable_api_table_conditional_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1, state=False) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1, state=False) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_column_family_name(): - # [START bigtable_api_column_family_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_families = table.list_column_families() - column_family_obj = column_families[COLUMN_FAMILY_ID] - column_family_name = column_family_obj.name - # [END bigtable_api_column_family_name] - import re - - _cf_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/tables/" - r"(?P[^/]+)/columnFamilies/" - r"(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _cf_name_re.match(column_family_name) - - -def test_bigtable_create_update_delete_column_family(): - # [START bigtable_api_create_column_family] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_id = "column_family_id1" - gc_rule = column_family.MaxVersionsGCRule(2) - column_family_obj = table.column_family(column_family_id, gc_rule=gc_rule) - column_family_obj.create() - - # [END bigtable_api_create_column_family] - column_families = table.list_column_families() - assert column_families[column_family_id].gc_rule == gc_rule - - # [START bigtable_api_update_column_family] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Already existing column family id - column_family_id = "column_family_id1" - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - column_family_obj = table.column_family(column_family_id, gc_rule=max_age_rule) - column_family_obj.update() - # [END bigtable_api_update_column_family] - - updated_families = table.list_column_families() - assert updated_families[column_family_id].gc_rule == max_age_rule - - # [START bigtable_api_delete_column_family] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_id = "column_family_id1" - column_family_obj = table.column_family(column_family_id) - column_family_obj.delete() - # [END bigtable_api_delete_column_family] - column_families = table.list_column_families() - assert column_family_id not in column_families - - -def test_bigtable_add_row_add_row_range_add_row_range_from_keys(): - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_5", - b"row_key_6", - b"row_key_7", - b"row_key_8", - b"row_key_9", - ] - - rows = [] - for row_key in row_keys: - row = Config.TABLE.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - rows.append(row) - Config.TABLE.mutate_rows(rows) - - # [START bigtable_api_add_row_key] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_key(b"row_key_5") - # [END bigtable_api_add_row_key] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_5"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - - # [START bigtable_api_add_row_range] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowRange, RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range(RowRange(start_key=b"row_key_3", end_key=b"row_key_7")) - # [END bigtable_api_add_row_range] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_3", b"row_key_4", b"row_key_5", b"row_key_6"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - - # [START bigtable_api_row_range_from_keys] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range_from_keys(start_key=b"row_key_3", end_key=b"row_key_7") - # [END bigtable_api_row_range_from_keys] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_3", b"row_key_4", b"row_key_5", b"row_key_6"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - table.truncate(timeout=200) - - -def test_bigtable_add_row_range_with_prefix(): - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"sample_row_key_1", - b"sample_row_key_2", - ] - - rows = [] - for row_key in row_keys: - row = Config.TABLE.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - rows.append(row) - Config.TABLE.mutate_rows(rows) - - # [START bigtable_api_add_row_range_with_prefix] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range_with_prefix("row") - # [END bigtable_api_add_row_range_with_prefix] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - ] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - table.truncate(timeout=200) - - -def test_bigtable_batcher_mutate_flush_mutate_rows(): - # [START bigtable_api_batcher_mutate] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - # Batcher for max row bytes, max_row_bytes=1024 is optional. - batcher = table.mutations_batcher(max_row_bytes=1024) - - # Add a single row - row_key = b"row_key_1" - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, "value-0", timestamp=datetime.now(timezone.utc) - ) - - # In batcher, mutate will flush current batch if it - # reaches the max_row_bytes - batcher.mutate(row) - batcher.flush() - # [END bigtable_api_batcher_mutate] - - # [START bigtable_api_batcher_flush] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - # Batcher for max row bytes, max_row_bytes=1024 is optional. - batcher = table.mutations_batcher(max_row_bytes=1024) - - # Add a single row - row_key = b"row_key" - row = table.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, "value-0") - - # In batcher, mutate will flush current batch if it - # reaches the max_row_bytes - batcher.mutate(row) - batcher.flush() - # [END bigtable_api_batcher_flush] - - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == 2 - table.truncate(timeout=200) - - # [START bigtable_api_batcher_mutate_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - batcher = table.mutations_batcher() - - row1 = table.row(b"row_key_1") - row2 = table.row(b"row_key_2") - row3 = table.row(b"row_key_3") - row4 = table.row(b"row_key_4") - - row1.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val1") - row2.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val2") - row3.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val3") - row4.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val4") - - batcher.mutate_rows([row1, row2, row3, row4]) - - # batcher will flush current batch if it - # reaches the max flush_count - # Manually send the current batch to Cloud Bigtable - batcher.flush() - # [END bigtable_api_batcher_mutate_rows] - - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == 4 - table.truncate(timeout=200) - - -def test_bigtable_create_family_gc_max_age(): - # [START bigtable_api_create_family_gc_max_age] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - column_family_obj = table.column_family("cf1", max_age_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_max_age] - rule = str(column_family_obj.to_pb()) - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_max_versions(): - # [START bigtable_api_create_family_gc_max_versions] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - - column_family_obj = table.column_family("cf2", max_versions_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_max_versions] - rule = str(column_family_obj.to_pb()) - assert "max_num_versions: 2" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_union(): - # [START bigtable_api_create_family_gc_union] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - max_versions_rule = column_family.MaxVersionsGCRule(2) - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - union_rule = column_family.GCRuleUnion([max_versions_rule, max_age_rule]) - - column_family_obj = table.column_family("cf3", union_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_union] - rule = str(column_family_obj.to_pb()) - assert "union" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - assert "max_num_versions: 2" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_intersection(): - # [START bigtable_api_create_family_gc_intersection] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - max_versions_rule = column_family.MaxVersionsGCRule(2) - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - intersection_rule = column_family.GCRuleIntersection( - [max_versions_rule, max_age_rule] - ) - - column_family_obj = table.column_family("cf4", intersection_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_intersection] - - rule = str(column_family_obj.to_pb()) - assert "intersection" in rule - assert "max_num_versions: 2" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_nested(): - # [START bigtable_api_create_family_gc_nested] - from google.cloud.bigtable import Client, column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Create a column family with nested GC policies. - # Create a nested GC rule: - # Drop cells that are either older than the 10 recent versions - # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - rule1 = column_family.MaxVersionsGCRule(10) - rule2 = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) - - nested_rule = column_family.GCRuleUnion([rule1, rule2]) - - column_family_obj = table.column_family("cf5", nested_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_nested] - - rule = str(column_family_obj.to_pb()) - assert "intersection" in rule - assert "max_num_versions: 2" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_row_data_cells_cell_value_cell_values(): - value = b"value_in_col1" - row = Config.TABLE.row(b"row_key_1") - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, value, timestamp=datetime.now(timezone.utc) - ) - row.commit() - - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, value, timestamp=datetime.now(timezone.utc) - ) - row.commit() - - # [START bigtable_api_row_data_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cells = row_data.cells - # [END bigtable_api_row_data_cells] - - actual_cell_value = cells[COLUMN_FAMILY_ID][COL_NAME1][0].value - assert actual_cell_value == value - - # [START bigtable_api_row_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cell_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - # [END bigtable_api_row_cell_value] - assert cell_value == value - - # [START bigtable_api_row_cell_values] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cell_values = row_data.cell_values(COLUMN_FAMILY_ID, COL_NAME1) - # [END bigtable_api_row_cell_values] - - for actual_value, timestamp in cell_values: - assert actual_value == value - - value2 = b"value_in_col2" - row.set_cell(COLUMN_FAMILY_ID, COL_NAME2, value2) - row.commit() - - # [START bigtable_api_row_find_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row = table.read_row(row_key) - - cells = row.find_cells(COLUMN_FAMILY_ID, COL_NAME2) - # [END bigtable_api_row_find_cells] - - assert cells[0].value == value2 - table.truncate(timeout=200) - - -def test_bigtable_row_setcell_rowkey(): - # [START bigtable_api_row_set_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY1) - - cell_val = b"cell-val" - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, cell_val, timestamp=datetime.now(timezone.utc) - ) - # [END bigtable_api_row_set_cell] - - response = table.mutate_rows([row]) - # validate that all rows written successfully - for i, status in enumerate(response): - assert status.code == 0 - - # [START bigtable_api_row_row_key] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row = table.row(ROW_KEY1) - row_key = row.row_key - # [END bigtable_api_row_row_key] - assert row_key == ROW_KEY1 - - # [START bigtable_api_row_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row = table.row(ROW_KEY1) - table1 = row.table - # [END bigtable_api_row_table] - - assert table1 == table - table.truncate(timeout=200) - - -def test_bigtable_row_delete(): - table_row_del = Config.INSTANCE.table(TABLE_ID) - row_obj = table_row_del.row(b"row_key_1") - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - row_obj.commit() - written_row_keys = [] - for row in table_row_del.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [b"row_key_1"] - - # [START bigtable_api_row_delete] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete() - row_obj.commit() - # [END bigtable_api_row_delete] - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - assert len(written_row_keys) == 0 - - -def test_bigtable_row_delete_cell(): - table_row_del_cell = Config.INSTANCE.table(TABLE_ID) - row_key1 = b"row_key_1" - row_obj = table_row_del_cell.row(row_key1) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row_obj.commit() - - written_row_keys = [] - for row in table_row_del_cell.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [row_key1] - - # [START bigtable_api_row_delete_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete_cell(COLUMN_FAMILY_ID, COL_NAME1) - row_obj.commit() - # [END bigtable_api_row_delete_cell] - - for row in table.read_rows(): - assert not row.row_key - - -def test_bigtable_row_delete_cells(): - table_row_del_cells = Config.INSTANCE.table(TABLE_ID) - row_key1 = b"row_key_1" - row_obj = table_row_del_cells.row(row_key1) - - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row_obj.commit() - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME2, CELL_VAL2) - row_obj.commit() - - written_row_keys = [] - for row in table_row_del_cells.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [row_key1] - - # [START bigtable_api_row_delete_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete_cells(COLUMN_FAMILY_ID, [COL_NAME1, COL_NAME2]) - row_obj.commit() - # [END bigtable_api_row_delete_cells] - - for row in table.read_rows(): - assert not row.row_key - - -def test_bigtable_row_clear(): - table_row_clear = Config.INSTANCE.table(TABLE_ID) - row_obj = table_row_clear.row(b"row_key_1") - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - - mutation_size = row_obj.get_mutations_size() - assert mutation_size > 0 - - # [START bigtable_api_row_clear] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - - row_obj.clear() - # [END bigtable_api_row_clear] - - mutation_size = row_obj.get_mutations_size() - assert mutation_size == 0 - - -def test_bigtable_row_clear_get_mutations_size(): - # [START bigtable_api_row_get_mutations_size] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key_id = b"row_key_1" - row_obj = table.row(row_key_id) - - mutation_size = row_obj.get_mutations_size() - # [END bigtable_api_row_get_mutations_size] - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - mutation_size = row_obj.get_mutations_size() - assert mutation_size > 0 - - row_obj.clear() - mutation_size = row_obj.get_mutations_size() - assert mutation_size == 0 - - -def test_bigtable_row_setcell_commit_rowkey(): - # [START bigtable_api_row_set_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - cell_val = b"cell-val" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - # [END bigtable_api_row_set_cell] - row_obj.commit() - - # [START bigtable_api_row_commit] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_2" - cell_val = b"cell-val" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - row_obj.commit() - # [END bigtable_api_row_commit] - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == [b"row_key_1", b"row_key_2"] - - # [START bigtable_api_row_row_key] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key_id = b"row_key_2" - row_obj = table.row(row_key_id) - row_key = row_obj.row_key - # [END bigtable_api_row_row_key] - assert row_key == row_key_id - table.truncate(timeout=300) - - -def test_bigtable_row_append_cell_value(): - row = Config.TABLE.row(ROW_KEY1) - - cell_val1 = b"1" - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val1) - row.commit() - - # [START bigtable_api_row_append_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY1, append=True) - - cell_val2 = b"2" - row.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, cell_val2) - # [END bigtable_api_row_append_cell_value] - row.commit() - - row_data = table.read_row(ROW_KEY1) - actual_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - assert actual_value == cell_val1 + cell_val2 - - # [START bigtable_api_row_commit] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = Config.TABLE.row(ROW_KEY2) - cell_val = 1 - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - row.commit() - # [END bigtable_api_row_commit] - - # [START bigtable_api_row_increment_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY2, append=True) - - int_val = 3 - row.increment_cell_value(COLUMN_FAMILY_ID, COL_NAME1, int_val) - # [END bigtable_api_row_increment_cell_value] - row.commit() - - row_data = table.read_row(ROW_KEY2) - actual_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - - import struct - - _PACK_I64 = struct.Struct(">q").pack - assert actual_value == _PACK_I64(cell_val + int_val) - table.truncate(timeout=200) - - -if __name__ == "__main__": - pytest.main() diff --git a/packages/google-cloud-bigtable/docs/classic_client/table-api.rst b/packages/google-cloud-bigtable/docs/classic_client/table-api.rst deleted file mode 100644 index 1bbf851462bb..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/table-api.rst +++ /dev/null @@ -1,153 +0,0 @@ -Table Admin API -=============== - -After creating an :class:`Instance `, you can -interact with individual tables, groups of tables or column families within -a table. - -List Tables ------------ - -If you want a comprehensive list of all existing tables in a instance, make a -`ListTables`_ API request with -:meth:`Instance.list_tables() `: - -.. code:: python - - >>> instance.list_tables() - [, - ] - -Table Factory -------------- - -To create a :class:`Table ` object: - -.. code:: python - - table = instance.table(table_id) - -Even if this :class:`Table ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`ColumnFamily ` -or :class:`Row `. - -Create a new Table ------------------- - -After creating the table object, make a `CreateTable`_ API request -with :meth:`create() `: - -.. code:: python - - table.create() - -If you would like to initially split the table into several tablets (tablets are -similar to HBase regions): - -.. code:: python - - table.create(initial_split_keys=['s1', 's2']) - -Delete an existing Table ------------------------- - -Make a `DeleteTable`_ API request with -:meth:`delete() `: - -.. code:: python - - table.delete() - -List Column Families in a Table -------------------------------- - -Though there is no **official** method for retrieving `column families`_ -associated with a table, the `GetTable`_ API method returns a -table object with the names of the column families. - -To retrieve the list of column families use -:meth:`list_column_families() `: - -.. code:: python - - column_families = table.list_column_families() - -Column Family Factory ---------------------- - -To create a -:class:`ColumnFamily ` object: - -.. code:: python - - column_family = table.column_family(column_family_id) - -There is no real reason to use this factory unless you intend to -create or delete a column family. - -In addition, you can specify an optional ``gc_rule`` (a -:class:`GarbageCollectionRule ` -or similar): - -.. code:: python - - column_family = table.column_family(column_family_id, - gc_rule=gc_rule) - -This rule helps the backend determine when and how to clean up old cells -in the column family. - -See :doc:`column-family` for more information about -:class:`GarbageCollectionRule ` -and related classes. - -Create a new Column Family --------------------------- - -After creating the column family object, make a `CreateColumnFamily`_ API -request with -:meth:`ColumnFamily.create() ` - -.. code:: python - - column_family.create() - -Delete an existing Column Family --------------------------------- - -Make a `DeleteColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.delete() - -Update an existing Column Family --------------------------------- - -Make an `UpdateColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.update() - -Next Step ---------- - -Now we go down the final step of the hierarchy from -:class:`Table ` to -:class:`Row ` as well as streaming -data directly via a :class:`Table `. - -Head next to learn about the :doc:`data-api`. - -.. _ListTables: https://googleapis.dev/python/bigtable/latest/table-api.html#list-tables -.. _CreateTable: https://googleapis.dev/python/bigtable/latest/table-api.html#create-a-new-table -.. _DeleteTable: https://googleapis.dev/python/bigtable/latest/table-api.html#delete-an-existing-table -.. _GetTable: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_table_admin.proto#L97-L102 -.. _CreateColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#create-a-new-column-family -.. _UpdateColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#update-an-existing-column-family -.. _DeleteColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#delete-an-existing-column-family -.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers diff --git a/packages/google-cloud-bigtable/docs/classic_client/table.rst b/packages/google-cloud-bigtable/docs/classic_client/table.rst deleted file mode 100644 index c230725d1351..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/table.rst +++ /dev/null @@ -1,6 +0,0 @@ -Table -~~~~~ - -.. automodule:: google.cloud.bigtable.table - :members: - :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/classic_client/usage.rst b/packages/google-cloud-bigtable/docs/classic_client/usage.rst deleted file mode 100644 index 7a47f4d4a418..000000000000 --- a/packages/google-cloud-bigtable/docs/classic_client/usage.rst +++ /dev/null @@ -1,38 +0,0 @@ -Classic Client -============== - -.. toctree:: - :maxdepth: 2 - - client-intro - - instance-api - table-api - data-api - - client - cluster - instance - table - app-profile - backup - column-family - encryption-info - row - row-data - row-filters - row-set - batcher - - -In the hierarchy of API concepts - -* a :class:`Client ` owns an - :class:`Instance ` -* an :class:`Instance ` owns a - :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily ` -* a :class:`Table ` owns a - :class:`Row ` - (and all the cells in the row) diff --git a/packages/google-cloud-bigtable/docs/data_client/async_data_authorized_view.rst b/packages/google-cloud-bigtable/docs/data_client/async_data_authorized_view.rst index 7d731297049f..5cd9c07b426b 100644 --- a/packages/google-cloud-bigtable/docs/data_client/async_data_authorized_view.rst +++ b/packages/google-cloud-bigtable/docs/data_client/async_data_authorized_view.rst @@ -6,6 +6,6 @@ Authorized View Async It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's performance benefits, the codebase should be designed to be async from the ground up. -.. autoclass:: google.cloud.bigtable.data._async.client.AuthorizedViewAsync +.. autoclass:: google.cloud.bigtable.data.AuthorizedViewAsync :members: :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/data_client/async_data_materialized_view.rst b/packages/google-cloud-bigtable/docs/data_client/async_data_materialized_view.rst index 5392400a2ce9..7b5192c34834 100644 --- a/packages/google-cloud-bigtable/docs/data_client/async_data_materialized_view.rst +++ b/packages/google-cloud-bigtable/docs/data_client/async_data_materialized_view.rst @@ -6,6 +6,6 @@ Materialized View Async It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's performance benefits, the codebase should be designed to be async from the ground up. -.. autoclass:: google.cloud.bigtable.data._async.client.MaterializedViewAsync +.. autoclass:: google.cloud.bigtable.data.MaterializedViewAsync :members: :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/data_client/async_data_mutations_batcher.rst b/packages/google-cloud-bigtable/docs/data_client/async_data_mutations_batcher.rst index 3e81f885a338..e79dfd6c3f2c 100644 --- a/packages/google-cloud-bigtable/docs/data_client/async_data_mutations_batcher.rst +++ b/packages/google-cloud-bigtable/docs/data_client/async_data_mutations_batcher.rst @@ -1,6 +1,6 @@ Mutations Batcher Async ~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: google.cloud.bigtable.data._async.mutations_batcher +.. autoclass:: google.cloud.bigtable.data.MutationsBatcherAsync :members: :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/data_client/async_data_table.rst b/packages/google-cloud-bigtable/docs/data_client/async_data_table.rst index 37c396570fba..231b933f157f 100644 --- a/packages/google-cloud-bigtable/docs/data_client/async_data_table.rst +++ b/packages/google-cloud-bigtable/docs/data_client/async_data_table.rst @@ -6,6 +6,6 @@ Table Async It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's performance benefits, the codebase should be designed to be async from the ground up. -.. autoclass:: google.cloud.bigtable.data._async.client.TableAsync +.. autoclass:: google.cloud.bigtable.data.TableAsync :members: :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/data_client/sync_data_authorized_view.rst b/packages/google-cloud-bigtable/docs/data_client/sync_data_authorized_view.rst index c0ac29721d5d..012053b57da1 100644 --- a/packages/google-cloud-bigtable/docs/data_client/sync_data_authorized_view.rst +++ b/packages/google-cloud-bigtable/docs/data_client/sync_data_authorized_view.rst @@ -1,6 +1,6 @@ Authorized View ~~~~~~~~~~~~~~~ -.. autoclass:: google.cloud.bigtable.data._sync_autogen.client.AuthorizedView +.. autoclass:: google.cloud.bigtable.data.AuthorizedView :members: :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/data_client/sync_data_materialized_view.rst b/packages/google-cloud-bigtable/docs/data_client/sync_data_materialized_view.rst index 0ae924182386..232469a3e52c 100644 --- a/packages/google-cloud-bigtable/docs/data_client/sync_data_materialized_view.rst +++ b/packages/google-cloud-bigtable/docs/data_client/sync_data_materialized_view.rst @@ -1,6 +1,6 @@ Materialized View ~~~~~~~~~~~~~~~~~ -.. autoclass:: google.cloud.bigtable.data._sync_autogen.client.MaterializedView +.. autoclass:: google.cloud.bigtable.data.MaterializedView :members: :inherited-members: diff --git a/packages/google-cloud-bigtable/docs/data_client/sync_data_mutations_batcher.rst b/packages/google-cloud-bigtable/docs/data_client/sync_data_mutations_batcher.rst index 2b7d1bfe094d..8e94b2bb5d71 100644 --- a/packages/google-cloud-bigtable/docs/data_client/sync_data_mutations_batcher.rst +++ b/packages/google-cloud-bigtable/docs/data_client/sync_data_mutations_batcher.rst @@ -1,6 +1,6 @@ Mutations Batcher ~~~~~~~~~~~~~~~~~ -.. automodule:: google.cloud.bigtable.data._sync_autogen.mutations_batcher +.. autoclass:: google.cloud.bigtable.data.MutationsBatcher :members: :show-inheritance: diff --git a/packages/google-cloud-bigtable/docs/index.rst b/packages/google-cloud-bigtable/docs/index.rst index 1c6a396f6090..16cfe87fd794 100644 --- a/packages/google-cloud-bigtable/docs/index.rst +++ b/packages/google-cloud-bigtable/docs/index.rst @@ -9,7 +9,6 @@ Client Types :maxdepth: 3 data_client/data_client_usage - classic_client/usage admin_client/admin_client_usage diff --git a/packages/google-cloud-bigtable/docs/scripts/patch_devsite_toc.py b/packages/google-cloud-bigtable/docs/scripts/patch_devsite_toc.py deleted file mode 100644 index 27fbd0cca7aa..000000000000 --- a/packages/google-cloud-bigtable/docs/scripts/patch_devsite_toc.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script will run after ``nox -s docfx`` is run. docfx is the api doc format used by -google cloud. It is described here: https://github.com/googleapis/docuploader?tab=readme-ov-file#requirements-for-docfx-yaml-tarballs. - -One of the file used by docfx is toc.yml which is used to generate the table of contents sidebar. -This script will patch file to create subfolders for each of the clients -""" - -import glob -import os -import shutil - -import yaml - -# set working directory to /docs -os.chdir(f"{os.path.dirname(os.path.abspath(__file__))}/{os.pardir}") - - -def add_sections(toc_file_path, section_list, output_file_path=None): - """ - Add new sections to the autogenerated docfx table of contents file - - Takes in a list of TocSection objects, which should point to a directory of rst files - within the main /docs directory, which represents a self-contained section of content - - :param toc_file_path: path to the autogenerated toc file - :param section_list: list of TocSection objects to add - :param output_file_path: path to save the updated toc file. If None, save to the input file - """ - # remove any sections that are already in the toc - remove_sections(toc_file_path, [section.title for section in section_list]) - # add new sections - current_toc = yaml.safe_load(open(toc_file_path, "r")) - for section in section_list: - print(f"Adding section {section.title}...") - current_toc[0]["items"].insert(-1, section.to_dict()) - section.copy_markdown() - # save file - if output_file_path is None: - output_file_path = toc_file_path - with open(output_file_path, "w") as f: - yaml.dump(current_toc, f) - - -def remove_sections(toc_file_path, section_list, output_file_path=None): - """ - Remove sections from the autogenerated docfx table of contents file - - Takes in a list of string section names to remove from the toc file - - :param toc_file_path: path to the autogenerated toc file - :param section_list: list of section names to remove - :param output_file_path: path to save the updated toc file. If None, save to the input file - """ - current_toc = yaml.safe_load(open(toc_file_path, "r")) - print(f"Removing sections {section_list}...") - new_items = [d for d in current_toc[0]["items"] if d["name"] not in section_list] - current_toc[0]["items"] = new_items - # save file - if output_file_path is None: - output_file_path = toc_file_path - with open(output_file_path, "w") as f: - yaml.dump(current_toc, f) - - -class TocSection: - def __init__(self, dir_name, index_file_name): - """ - :param dir_name: name of the directory containing the rst files - :param index_file_name: name of an index file within dir_name. This file - will not be included in the table of contents, but provides an ordered - list of the other files which should be included - """ - self.dir_name = dir_name - self.index_file_name = index_file_name - index_file_path = os.path.join(dir_name, index_file_name) - # find set of files referenced by the index file - with open(index_file_path, "r") as f: - self.title = None - in_toc = False - self.items = [] - for line in f: - # ignore empty lines - if not line.strip(): - continue - # add files explictly included in the toc - if line.startswith(".. include::"): - file_base = os.path.splitext(line.split("::")[1].strip())[0] - self.items.append( - self.extract_toc_entry( - file_base, file_title=file_base.capitalize() - ) - ) - continue - if line.startswith(".. toctree::"): - in_toc = True - continue - # ignore directives - if ":" in line: - continue - # set tile as first line with no directive - if self.title is None: - self.title = line.strip() - if not in_toc: - continue - # bail when toc indented block is done - if not line.startswith(" ") and not line.startswith("\t"): - in_toc = False - continue - # extract entries - self.items.append(self.extract_toc_entry(line.strip())) - - def extract_toc_entry(self, file_name, file_title=None): - """ - Given the name of a file, extract the title and href for the toc entry, - and return as a dictionary - """ - # load the file to get the title - with open(f"{self.dir_name}/{file_name}.rst", "r") as f2: - if file_title is None: - # use first line as title if not provided - file_title = f2.readline().strip() - return {"name": file_title, "href": f"{file_name}.md"} - - def to_dict(self): - """ - Convert the TocSection object to a dictionary that can be written to a yaml file - """ - return {"name": self.title, "items": self.items} - - def copy_markdown(self): - """ - Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml - - This is necessary because the markdown files in sub-directories - are not copied over by the docfx build by default - """ - for file in os.listdir("_build/markdown/" + self.dir_name): - shutil.copy( - f"_build/markdown/{self.dir_name}/{file}", - f"_build/html/docfx_yaml", - ) - - def validate_section(self, toc): - # Make sure each rst file is listed in the toc. - items_in_toc = [ - d["items"] for d in toc[0]["items"] if d["name"] == self.title and ".rst" - ][0] - items_in_dir = [f for f in os.listdir(self.dir_name) if f.endswith(".rst")] - # subtract 1 for index - assert len(items_in_toc) == len(items_in_dir) - 1 - for file in items_in_dir: - if file != self.index_file_name: - base_name, _ = os.path.splitext(file) - assert any(d["href"] == f"{base_name}.md" for d in items_in_toc) - # make sure the markdown files are present in the docfx_yaml directory - md_files = [d["href"] for d in items_in_toc] - for file in md_files: - assert os.path.exists(f"_build/html/docfx_yaml/{file}") - - -class UIDFilteredTocSection(TocSection): - def __init__(self, toc_file_path, section_name, title, uid_prefix): - """Creates a filtered section denoted by section_name in the toc_file_path to items with the given UID prefix. - - The section is then renamed to the title. - """ - current_toc = yaml.safe_load(open(toc_file_path, "r")) - self.uid_prefix = uid_prefix - - # Since we are looking for a specific section_name there should only - # be one match. - section_items = [ - d for d in current_toc[0]["items"] if d["name"] == section_name - ][0]["items"] - filtered_items = [d for d in section_items if d["uid"].startswith(uid_prefix)] - self.items = filtered_items - self.title = title - - def copy_markdown(self): - """ - No-op because we are filtering on UIDs, not markdown files. - """ - pass - - def validate_section(self, toc): - uids_in_toc = set() - - # A UID-filtered TOC tree looks like the following: - # - items: - # items: - # name: - # uid: - # - # Walk through the TOC tree to find all UIDs recursively. - def find_uids_in_items(items): - uids_in_toc.add(items["uid"]) - for subitem in items.get("items", []): - find_uids_in_items(subitem) - - items_in_toc = [d["items"] for d in toc[0]["items"] if d["name"] == self.title][ - 0 - ] - for item in items_in_toc: - find_uids_in_items(item) - - # Now that we have all the UIDs, first match all of them - # with corresponding .yml files. - for uid in uids_in_toc: - assert os.path.exists(f"_build/html/docfx_yaml/{uid}.yml") - - # Also validate that every uid yml file that starts with the uid_prefix - # exists in the section. - for filename in glob.glob( - f"{self.uid_prefix}*.yml", root_dir="_build/html/docfx_yaml" - ): - assert filename[:-4] in uids_in_toc - - -def validate_toc(toc_file_path, expected_section_list, added_sections): - current_toc = yaml.safe_load(open(toc_file_path, "r")) - # make sure the set of sections matches what we expect - found_sections = [d["name"] for d in current_toc[0]["items"]] - assert found_sections == expected_section_list, ( - f"Expected {expected_section_list}, found {found_sections}" - ) - # make sure each customs ection is in the toc - for section in added_sections: - assert section.title in found_sections - section.validate_section(current_toc) - print("Toc validation passed") - - -if __name__ == "__main__": - # Add secrtions for the async_data_client and classic_client directories - toc_path = "_build/html/docfx_yaml/toc.yml" - - custom_sections = [ - TocSection(dir_name="data_client", index_file_name="data_client_usage.rst"), - UIDFilteredTocSection( - toc_file_path=toc_path, - section_name="Bigtable Admin V2", - title="Admin Client", - uid_prefix="google.cloud.bigtable_admin_v2", - ), - TocSection(dir_name="classic_client", index_file_name="usage.rst"), - ] - add_sections(toc_path, custom_sections) - # Remove the Bigtable section, since it has duplicated data - remove_sections(toc_path, ["Bigtable", "Bigtable Admin V2"]) - # run validation to make sure yaml is structured as we expect - validate_toc( - toc_file_path=toc_path, - expected_section_list=[ - "Overview", - "bigtable APIs", - "Changelog", - "Multiprocessing", - "Data Client", - "Admin Client", - "Classic Client", - ], - added_sections=custom_sections, - ) diff --git a/packages/google-cloud-bigtable/noxfile.py b/packages/google-cloud-bigtable/noxfile.py index 88d0237720fa..045dcc166a38 100644 --- a/packages/google-cloud-bigtable/noxfile.py +++ b/packages/google-cloud-bigtable/noxfile.py @@ -512,9 +512,6 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - # Customization: Add extra sections to the table of contents for the Classic vs Async clients - session.install("pyyaml") - session.run("python", "docs/scripts/patch_devsite_toc.py") @nox.session(python=DEFAULT_PYTHON_VERSION)