Skip to content

Commit aab9e58

Browse files
sasha-gitgsasha-gitg
authored
feat: add custom and hp tuning (#388)
1 parent 22409c3 commit aab9e58

16 files changed

+2501
-1046
lines changed

‎google/cloud/aiplatform/__init__.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,15 @@
2626
TimeSeriesDataset,
2727
VideoDataset,
2828
)
29+
from google.cloud.aiplatform import hyperparameter_tuning
30+
from google.cloud.aiplatform.metadata import metadata
2931
from google.cloud.aiplatform.models import Endpoint
3032
from google.cloud.aiplatform.models import Model
31-
from google.cloud.aiplatform.jobs import BatchPredictionJob
33+
from google.cloud.aiplatform.jobs import (
34+
BatchPredictionJob,
35+
CustomJob,
36+
HyperparameterTuningJob,
37+
)
3238
from google.cloud.aiplatform.training_jobs import (
3339
CustomTrainingJob,
3440
CustomContainerTrainingJob,
@@ -39,7 +45,6 @@
3945
AutoMLTextTrainingJob,
4046
AutoMLVideoTrainingJob,
4147
)
42-
from google.cloud.aiplatform.metadata import metadata
4348

4449
"""
4550
Usage:
@@ -60,6 +65,7 @@
6065
"explain",
6166
"gapic",
6267
"init",
68+
"hyperparameter_tuning",
6369
"log_params",
6470
"log_metrics",
6571
"get_experiment_df",
@@ -71,11 +77,13 @@
7177
"AutoMLTextTrainingJob",
7278
"AutoMLVideoTrainingJob",
7379
"BatchPredictionJob",
80+
"CustomJob",
7481
"CustomTrainingJob",
7582
"CustomContainerTrainingJob",
7683
"CustomPythonPackageTrainingJob",
7784
"Endpoint",
7885
"ImageDataset",
86+
"HyperparameterTuningJob",
7987
"Model",
8088
"TabularDataset",
8189
"TextDataset",

‎google/cloud/aiplatform/base.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,29 @@ def log_create_complete(
101101
f"{variable_name} = aiplatform.{cls.__name__}('{resource.name}')"
102102
)
103103

104+
def log_create_complete_with_getter(
105+
self,
106+
cls: Type["AiPlatformResourceNoun"],
107+
resource: proto.Message,
108+
variable_name: str,
109+
):
110+
"""Logs create event is complete.
111+
112+
Will also include code snippet to instantiate resource in SDK.
113+
114+
Args:
115+
cls (AiPlatformResourceNoun):
116+
AI Platform Resource Noun class that is being created.
117+
resource (proto.Message):
118+
AI Platform Resourc proto.Message
119+
variable_name (str): Name of variable to use for code snippet
120+
"""
121+
self._logger.info(f"{cls.__name__} created. Resource name: {resource.name}")
122+
self._logger.info(f"To use this {cls.__name__} in another session:")
123+
self._logger.info(
124+
f"{variable_name} = aiplatform.{cls.__name__}.get('{resource.name}')"
125+
)
126+
104127
def log_action_start_against_resource(
105128
self, action: str, noun: str, resource_noun_obj: "AiPlatformResourceNoun"
106129
):
@@ -543,6 +566,11 @@ def update_time(self) -> datetime.datetime:
543566
self._sync_gca_resource()
544567
return self._gca_resource.update_time
545568

569+
@property
570+
def gca_resource(self) -> proto.Message:
571+
"""The underlying resource proto represenation."""
572+
return self._gca_resource
573+
546574
def __repr__(self) -> str:
547575
return f"{object.__repr__(self)} \nresource name: {self.resource_name}"
548576

‎google/cloud/aiplatform/compat/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
types.prediction_service = types.prediction_service_v1beta1
7171
types.specialist_pool = types.specialist_pool_v1beta1
7272
types.specialist_pool_service = types.specialist_pool_service_v1beta1
73+
types.study = types.study_v1beta1
7374
types.training_pipeline = types.training_pipeline_v1beta1
7475
types.metadata_service = types.metadata_service_v1beta1
7576
types.tensorboard_service = types.tensorboard_service_v1beta1
@@ -120,6 +121,7 @@
120121
types.prediction_service = types.prediction_service_v1
121122
types.specialist_pool = types.specialist_pool_v1
122123
types.specialist_pool_service = types.specialist_pool_service_v1
124+
types.study = types.study_v1
123125
types.training_pipeline = types.training_pipeline_v1
124126

125127
__all__ = (

‎google/cloud/aiplatform/compat/types/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
prediction_service as prediction_service_v1beta1,
5050
specialist_pool as specialist_pool_v1beta1,
5151
specialist_pool_service as specialist_pool_service_v1beta1,
52+
study as study_v1beta1,
5253
training_pipeline as training_pipeline_v1beta1,
5354
metadata_service as metadata_service_v1beta1,
5455
tensorboard_service as tensorboard_service_v1beta1,
@@ -90,6 +91,7 @@
9091
prediction_service as prediction_service_v1,
9192
specialist_pool as specialist_pool_v1,
9293
specialist_pool_service as specialist_pool_service_v1,
94+
study as study_v1,
9395
training_pipeline as training_pipeline_v1,
9496
)
9597

‎google/cloud/aiplatform/hyperparameter_tuning.py

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Copyright 2021 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
import abc
19+
from typing import Dict, List, Optional, Sequence, Tuple, Union
20+
21+
import proto
22+
23+
from google.cloud.aiplatform.compat.types import study as gca_study_compat
24+
25+
_SCALE_TYPE_MAP = {
26+
"linear": gca_study_compat.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
27+
"log": gca_study_compat.StudySpec.ParameterSpec.ScaleType.UNIT_LOG_SCALE,
28+
"reverse_log": gca_study_compat.StudySpec.ParameterSpec.ScaleType.UNIT_REVERSE_LOG_SCALE,
29+
"unspecified": gca_study_compat.StudySpec.ParameterSpec.ScaleType.SCALE_TYPE_UNSPECIFIED,
30+
}
31+
32+
33+
class _ParameterSpec(metaclass=abc.ABCMeta):
34+
"""Base class represents a single parameter to optimize."""
35+
36+
def __init__(
37+
self,
38+
conditional_parameter_spec: Optional[Dict[str, "_ParameterSpec"]] = None,
39+
parent_values: Optional[List[Union[float, int, str]]] = None,
40+
):
41+
42+
self.conditional_parameter_spec = conditional_parameter_spec
43+
self.parent_values = parent_values
44+
45+
@property
46+
@classmethod
47+
@abc.abstractmethod
48+
def _proto_parameter_value_class(self) -> proto.Message:
49+
"""The proto representation of this parameter."""
50+
pass
51+
52+
@property
53+
@classmethod
54+
@abc.abstractmethod
55+
def _parameter_value_map(self) -> Tuple[Tuple[str, str]]:
56+
"""A Tuple map of parameter key to underlying proto key."""
57+
pass
58+
59+
@property
60+
@classmethod
61+
@abc.abstractmethod
62+
def _parameter_spec_value_key(self) -> Tuple[Tuple[str, str]]:
63+
"""The ParameterSpec key this parameter should be assigned."""
64+
pass
65+
66+
@property
67+
def _proto_parameter_value_spec(self) -> proto.Message:
68+
"""Converts this parameter to it's parameter value representation."""
69+
proto_parameter_value_spec = self._proto_parameter_value_class()
70+
for self_attr_key, proto_attr_key in self._parameter_value_map:
71+
setattr(
72+
proto_parameter_value_spec, proto_attr_key, getattr(self, self_attr_key)
73+
)
74+
return proto_parameter_value_spec
75+
76+
def _to_parameter_spec(
77+
self, parameter_id: str
78+
) -> gca_study_compat.StudySpec.ParameterSpec:
79+
"""Converts this parameter to ParameterSpec."""
80+
# TODO: Conditional parameters
81+
parameter_spec = gca_study_compat.StudySpec.ParameterSpec(
82+
parameter_id=parameter_id,
83+
scale_type=_SCALE_TYPE_MAP.get(getattr(self, "scale", "unspecified")),
84+
)
85+
86+
setattr(
87+
parameter_spec,
88+
self._parameter_spec_value_key,
89+
self._proto_parameter_value_spec,
90+
)
91+
92+
return parameter_spec
93+
94+
95+
class DoubleParameterSpec(_ParameterSpec):
96+
97+
_proto_parameter_value_class = (
98+
gca_study_compat.StudySpec.ParameterSpec.DoubleValueSpec
99+
)
100+
_parameter_value_map = (("min", "min_value"), ("max", "max_value"))
101+
_parameter_spec_value_key = "double_value_spec"
102+
103+
def __init__(
104+
self, min: float, max: float, scale: str,
105+
):
106+
"""
107+
Value specification for a parameter in ``DOUBLE`` type.
108+
109+
Args:
110+
min (float):
111+
Required. Inclusive minimum value of the
112+
parameter.
113+
max (float):
114+
Required. Inclusive maximum value of the
115+
parameter.
116+
scale (str):
117+
Required. The type of scaling that should be applied to this parameter.
118+
119+
Accepts: 'linear', 'log', 'reverse_log'
120+
"""
121+
122+
super().__init__()
123+
124+
self.min = min
125+
self.max = max
126+
self.scale = scale
127+
128+
129+
class IntegerParameterSpec(_ParameterSpec):
130+
131+
_proto_parameter_value_class = (
132+
gca_study_compat.StudySpec.ParameterSpec.IntegerValueSpec
133+
)
134+
_parameter_value_map = (("min", "min_value"), ("max", "max_value"))
135+
_parameter_spec_value_key = "integer_value_spec"
136+
137+
def __init__(
138+
self, min: int, max: int, scale: str,
139+
):
140+
"""
141+
Value specification for a parameter in ``INTEGER`` type.
142+
143+
Args:
144+
min (float):
145+
Required. Inclusive minimum value of the
146+
parameter.
147+
max (float):
148+
Required. Inclusive maximum value of the
149+
parameter.
150+
scale (str):
151+
Required. The type of scaling that should be applied to this parameter.
152+
153+
Accepts: 'linear', 'log', 'reverse_log'
154+
"""
155+
156+
super().__init__()
157+
158+
self.min = min
159+
self.max = max
160+
self.scale = scale
161+
162+
163+
class CategoricalParameterSpec(_ParameterSpec):
164+
165+
_proto_parameter_value_class = (
166+
gca_study_compat.StudySpec.ParameterSpec.CategoricalValueSpec
167+
)
168+
_parameter_value_map = (("values", "values"),)
169+
_parameter_spec_value_key = "categorical_value_spec"
170+
171+
def __init__(
172+
self, values: Sequence[str],
173+
):
174+
"""Value specification for a parameter in ``CATEGORICAL`` type.
175+
176+
Args:
177+
values (Sequence[str]):
178+
Required. The list of possible categories.
179+
"""
180+
181+
super().__init__()
182+
183+
self.values = values
184+
185+
186+
class DiscreteParameterSpec(_ParameterSpec):
187+
188+
_proto_parameter_value_class = (
189+
gca_study_compat.StudySpec.ParameterSpec.DiscreteValueSpec
190+
)
191+
_parameter_value_map = (("values", "values"),)
192+
_parameter_spec_value_key = "discrete_value_spec"
193+
194+
def __init__(
195+
self, values: Sequence[float], scale: str,
196+
):
197+
"""Value specification for a parameter in ``DISCRETE`` type.
198+
199+
values (Sequence[float]):
200+
Required. A list of possible values.
201+
The list should be in increasing order and at
202+
least 1e-10 apart. For instance, this parameter
203+
might have possible settings of 1.5, 2.5, and
204+
4.0. This list should not contain more than
205+
1,000 values.
206+
scale (str):
207+
Required. The type of scaling that should be applied to this parameter.
208+
209+
Accepts: 'linear', 'log', 'reverse_log'
210+
"""
211+
212+
super().__init__()
213+
214+
self.values = values
215+
self.scale = scale

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy